#!/bin/bash
########################################################################
# script name:  tl7_upgrade_procedure.func
# Used for:
# Author:       Martin Dumont
# Contributors:
# Version:      0.1
#
# (date format: date +'%Y/%m/%d')
#
# Date          Version Contributors    Modifications
# ----------    ------- ------------    -------------
# 2012/05/02    0.1     MDUMONT         initial
########################################################################
#
# This function file is used to executes
# the steps in the TL7 upgrade document
#
########################################################################
#
# The order specify the execution order in the main script
# 1 will be executed before 2
# The value tells the main script two things:
#    1) the function to execute (be specific)
#    2) comment (documentation, for better understanding in the logs)
#       --- Do not put spaces ---
########################################################################
########################################################################
# Order_a array gives the right order of functions execution to the stepping script
# The first field is the function name and the second is a comment
########################################################################
Order_a=( 'A1:cfg2html and mksysb' 'A1_2:Verify EMC software' 'B1:Verify_current_AIX' 'B2:Verify_boot_image' 'B3:List_all_applied_but_uncommitted_fileset' 'B4:Commit all applied but uncommitted fileset' 'B5:Repeat listing of all applied but uncommitted fileset' 'B6:Verify the current integrity of the installed software' 'B7:Installing pre-requisite' 'B8:Preview AIX update' 'B9:Update AIX' 'B10:Install additionnal fileset' 'B11:Set Tivoli maintenance filters' 'B12:Shut down applications' 'B13:Change VSCSI and VIOC hdisk settings' 'B14:Unmount NFS-mounted filesystems' 'B15:Display bootlist' 'B16:Open terminal window on host being updated' 'B17:Reboot from HMC' 'B18:Verify that the new AIX' 'B19:Verify the integrity of the updated software' 'B20:Install extended software utilities' 'B21:Verify that VME is active' 'B22:Verify VSCSI and VIOC hdisk settings' 'B23:Change VMO and IO watermark settings' 'B24:Establish AIX Edition setting' 'B25:Change DMS timeout and RG monitor interval' 'B26_1:Re-create autoprobe' 'B26:Restart applications' 'B27:Remove Tivoli maintenance filters' 'B28:Syscheck' )
#######################################
# Variables
#######################################
#######################################
# Verification
#######################################
[[ ${UPGRADESOURCE?Error \$UPGRADESOURCE is not defined in $CONFFILE} ]]
[[ ${TARGETOSLEVEL?Error \$TARGETOSLEVEL is not defined in $CONFFILE} ]]
[[ ${LOGDIR?Error \$LOGDIR is not defined in $CONFFILE} ]]
if [ ! -d "$UPGRADESOURCE" ]; then /bin/echo "Directory $UPGRADESOURCE doesn't exist"; exit 1; fi
#######################################
CFG2HTMLDIR=/central/logs/cfg2html/`uname -n`
MKSYSBDIR=/central/mksysb/
MKSYSBLOG="$MKSYSBDIR/log/drmksysb.ksh_$(hostname | awk -F"." '{print $1}').log"
MKSYSBSCRIPT='/central/mksysb/script/drmksysb.ksh -m'
pre_requis=( '/central/depot/softwares/java/Java14_64.sdk.1.4.2.455/Java14_64.sdk.1.4.2.455.bff:Java14_64.sdk' '/central/aixcdrom/inhouse/6.1.0.0/inst.images/installp/ppc:Java5.sdk' '/central/aixcdrom/inhouse/6.1.0.0/inst.images/installp/ppc:bos.data' )
#######################################
# functions
#######################################
#######################################
function EXIT {
	rm $LOCK
	/bin/echo EXIT
}
function REBOOT {
	rm $LOCK
	/bin/echo REBOOT-NOW
}
#######################################
#######################################
function set_variables {
	LOCK=/tmp/tl7_upgrade.lock
	if [ -f $LOCK ]
	then
		/bin/echo "/tmp/tl7_upgrade.lock exist, please delete or verify if an upgrade is pending."
		exit 1
	else
		touch /tmp/tl7_upgrade.lock
	fi
	if [ ! -z "$pre_requis" ];
	then
	   x=0
	   while [ "$x" -lt "${#pre_requis[@]}" ]
	   do
	      DEVICE=$(/bin/echo ${pre_requis[$x]} | cut -d: -f1)
	      if [ ! -x "$DEVICE" ]; then echo "Directory for pre_requis[$x] doesn't exist ($DEVICE)"; exit 1; fi
	      ((x++))
	   done
	fi
   /bin/echo "+----------------------------------------------------------------------------------------------------------+"
   /bin/echo "+ Verifying oslevel: \c"
   OSLEVEL=$(oslevel -s)
   /bin/echo $OSLEVEL
   /bin/echo "+----------------------------------------------------------------------------------------------------------+"
   CURRENTBASE=$(/bin/echo $OSLEVEL | cut -d'-' -f1)
   CURRENTTL=$(/bin/echo $OSLEVEL | cut -d'-' -f2)
   CURRENTDATE=$(/bin/echo $OSLEVEL | cut -d'-' -f4)
   TARGETTL=$(/bin/echo $TARGETOSLEVEL | cut -d'-' -f2)
   TARGETBASE=$(/bin/echo $TARGETOSLEVEL | cut -d'-' -f1)
   TARGETDATE=$(/bin/echo $TARGETOSLEVEL | cut -d'-' -f4)

   #######################################

   cat <<-EOLEVEL

   Starting upgrade on `uname -n`

   Current oslevel:       $OSLEVEL
   Target oslevel:        $TARGETOSLEVEL
   Function file:         $FUNCFILE
   Logs directory:        $LOGDIR
   Check file:            $CHECKFILE
   Source filesets:       $UPGRADESOURCE

   Exclude: $EXCLUDE

	EOLEVEL
	if [ ! -d $LOGDIR ]; then mkdir -m 755 -p $LOGDIR || exit; fi
}
#######################################
#######################################
function A1_2 {
	/bin/echo "Verifying EMC software: \c"
	if lslpp -L | grep -q EMCpower.base
	then
		/bin/echo "found software"
		/bin/echo "\nPlease check in documentum for the upgrade procedure of EMC (EMC and ODM software upgrade with aix6.doc)\n"
		/bin/echo "Press any key to continue..."; read -n 1; /bin/echo
	else
		/bin/echo "nothing found"
	fi
}
#######################################
#######################################
function A1 {
	/bin/echo "Verifying cfg2html: \c"
	if [ -d "$CFG2HTMLDIR" ]
	then
		CFG2HTMLFILE=$CFG2HTMLDIR/$(ls -tr $CFG2HTMLDIR/ | grep "$(hostname -s)" | tail -1)
		if [ -f "$CFG2HTMLFILE" ]
		then
			FILEDATE=$(if [ -x /opt/freeware/bin/stat ]; then /opt/freeware/bin/stat -c %y $CFG2HTMLFILE | cut -d' ' -f1; fi)
			/bin/echo "$FILEDATE $CFG2HTMLFILE"
		else
			/bin/echo "not found"
		fi
	fi
	/bin/echo
	read -n 1 -p "Would you like to run a cfg2html for this host? [y/n]: "
	/bin/echo
	if [ "$REPLY" = "y" ]; then /var/code/sap/scripts/cfg2html_cron.sh; fi
	CONTINUE=no
	while [ "$CONTINUE" != "yes" ]
	do
		/bin/echo "Verifying mksysb: \c"
		MKSYSB=$MKSYSBDIR$(ls -tr $MKSYSBDIR | grep "$(hostname -s)" | grep 'mksysb$' | tail -1)
		if [ -f $MKSYSB ]
		then
			/bin/echo "Mksysb: $MKSYSB"
			/bin/echo
			/bin/lsmksysb -lf $MKSYSB
			/bin/echo
		else
			/bin/echo "none"
		fi
		/bin/echo
		read -n 1 -p "Would you like to run (or re-run) a mksysb on this host? [y/n]: "
		/bin/echo
		if [ "$REPLY" = "y" ]
		then
			/bin/echo "Starting a mksysb, this will take a moment..."
			/bin/echo "You can monitor the mksysb with this:"
			/bin/echo "tail -n +1 -f $MKSYSBLOG"
			/bin/echo "Executing mksysb..."
			$MKSYSBSCRIPT
			/bin/echo
			/bin/echo "If the mksysb is not good because of an error with bosboot, execute this script instead outside of this function:"
			/bin/echo /central/mksysb/script/drmksysb_bypass.ksh -m
			/bin/echo "Mksysb completed"
			/bin/echo
		else
			CONTINUE=yes
		fi
	done
}
#######################################
#######################################
function B1 {
	/bin/echo "Current oslevel: $OSLEVEL"
	if [ "$CURRENTBASE" != "$TARGETBASE" ]
	then 
		/bin/echo
		/bin/echo "Base version on the system needs to be at $TARGETBASE but is currently at $CURRENTBASE"
		/bin/echo
		EXIT
	fi
	if [ "$CURRENTTL" -ge "$TARGETTL" ]
	then 
		/bin/echo
		/bin/echo "TL level on the system is equal or higher then target level (current $CURRENTTL >= $TARGETTL)"
		/bin/echo
		EXIT
	fi
	if [ "$CURRENTDATE" -gt "$TARGETDATE" ]
	then
		/bin/echo
		/bin/echo "Target oslevel is older then the current level (current date $CURRENTDATE > $TARGETDATE)"
		/bin/echo
		EXIT
	fi
	/bin/echo; /bin/echo "Current oslevel on the system is ok to be upgraded, continuing..."
	sleep 1
	echo
}
#######################################
#######################################
function B2 {
	/bin/echo "Executing pre-commands: "
	set -x
	if ! bosboot -ad /dev/`bootinfo -b` ; then /bin/echo "Something went wrong, exiting"; EXIT; fi
	if ! bosboot -ad /dev/ipldevice ; then /bin/echo "Something went wrong, exiting"; EXIT; fi
	set +x
}
#######################################
#######################################
function B3 {
	if ! installp -s ; then /bin/echo "Something went wrong, exiting"; EXIT; fi
}
function B4 {
	if ! installp -cgX all ; then /bin/echo "Something went wrong, exiting"; EXIT; fi
}
function B5 {
	if ! installp -s ; then /bin/echo "Something went wrong, exiting"; EXIT; fi
}
function B6 {
	if ! lppchk -vm3 ; then /bin/echo "Something went wrong, exiting"; EXIT; fi
	if ! lslpp -L|grep -q devices.fcp.disk.ibm.mpio && ! lslpp -L|grep -q devices.sdd && ! ls -l /dev/ | grep -q vpath
	then
		/bin/echo
		/bin/echo No /dev/vpath found, continuing
		/bin/echo
	else
		/bin/echo
		/bin/rm /dev/vpath*
		/bin/echo
	fi
}
#######################################
#######################################
function B7 {
	x=0
	while [ "$x" -lt "${#pre_requis[@]}" ]
	do
		DEVICE=$(/bin/echo ${pre_requis[$x]} | cut -d: -f1)
		FILESET=$(/bin/echo ${pre_requis[$x]} | cut -d: -f2)
		/bin/echo "Installing pre_requis_$x:"
		if ! installp -acXYd $DEVICE $FILESET ; then /bin/echo; /bin/echo "Something went wrong, exiting"; EXIT; fi
		((x++))
	done
}
#######################################
#######################################
function B8 {
	if [ ! -d $UPGRADESOURCE ]; then /bin/echo "Directory $UPGRADESOURCE doesn't exist"; EXIT; fi
	echo /usr/lib/instl/sm_inst installp_cmd -a -d $UPGRADESOURCE -f '_update_all' -p -c -N -g -X -Y
	/usr/lib/instl/sm_inst installp_cmd -a -d $UPGRADESOURCE -f '_update_all' -p -c -N -g -X -Y
	read -n 1 -p "Looks ok? [y/n]: "
	/bin/echo
	if [ "$REPLY" != "y" ]
   then
      /bin/echo "--> Not ok...exiting"
      EXIT
   fi
}
#######################################
#######################################
function B9 {
	if [ ! -d $UPGRADESOURCE ]; then /bin/echo "Directory $UPGRADESOURCE doesn't exist"; EXIT; fi
	echo /usr/lib/instl/sm_inst installp_cmd -a -d $UPGRADESOURCE -f '_update_all' -c -N -g -X -Y
	/usr/lib/instl/sm_inst installp_cmd -a -d $UPGRADESOURCE -f '_update_all' -c -N -g -X -Y
	read -n 1 -p "Looks ok? [y/n]: "
	/bin/echo
	if [ "$REPLY" != "y" ]
   then
      /bin/echo "--> Not ok...exiting"
      EXIT
   fi
}
#######################################
#######################################
function B10 {
	x=0
	add_fileset=( '/central/aixcdrom/inhouse/aix6.1tl7sp2:mcr.rte' '/central/aixcdrom/inhouse/aix6.1tl5sp0:clic.rte.lib' )
	while [ "$x" -lt "${#add_fileset[@]}" ]
	do
		DEVICE=$(/bin/echo ${add_fileset[$x]} | cut -d: -f1)
		FILESET=$(/bin/echo ${add_fileset[$x]} | cut -d: -f2)
		/bin/echo "Installing add_fileset $FILESET:"
		if ! installp -acXFYd $DEVICE $FILESET ; then /bin/echo; /bin/echo "Something went wrong, exiting"; EXIT; fi
		((x++))
	done
	echo /usr/lib/instl/sm_inst installp_cmd -a -d $UPGRADESOURCE -f '_update_all' -c -N -g -X -Y
	/usr/lib/instl/sm_inst installp_cmd -a -d $UPGRADESOURCE -f '_update_all' -c -N -g -X -Y
	/bin/echo
	/bin/echo "Install efix IV16850m02"
	/bin/echo
	cd /central/aixcdrom/inhouse/IV16850m02 
	if ! emgr -X -e IV16850m02.120622.epkg.Z ; then /bin/echo; /bin/echo "Something went wrong, exiting"; EXIT; fi
}
#######################################
#######################################
function B11 {
	if [ -x /usr/local/scripts/settivolifilter ]
	then
		read -n 1 -p "Do you want to set the tivoli filter? [y/n]: "
		if [ "$REPLY" = "y" ]
	   then
			/usr/local/scripts/settivolifilter 60 `uname -n`	
  		fi
	else
		/bin/echo "Not setting tivoli filters, /usr/local/scripts/settivolifilter not found"
	fi
	/bin/echo
	/bin/echo "You can also go to Logon Tivoli Portal mwm (Maintenance Window Management) menu / Set Tivoli maintenance filters"
	/bin/echo 'https://mtllimon001.ca.aero.bombardier.net:9443/opview/login.jsp'
}
function B12 {
	read -n 1 -p "Are the applications down? [y/n]: "
	/bin/echo
	if [ "$REPLY" != "y" ]
   then
		/bin/echo "Please stop the apps"
		/bin/echo
		/bin/echo "Re-execute this script after you are done."
		/bin/echo
		EXIT
	fi
}
#######################################
#######################################
function B13 {
	lsdev | grep vscsi | grep Avail | awk '{print $1}' | while read line ; do echo $line ; chdev -l $line -a vscsi_err_recov=fast_fail -a vscsi_path_to=30 -P ; done
	lsdev | grep disk | grep Virtu | grep Avail | awk '{print $1}' | while read line ; do echo $line ; chdev -l $line -a queue_depth=20 -a hcheck_interval=60 -P ; done
	ioo -o pd_npages=32768 ;  /usr/sbin/tunsave -a -F nextboot -t ioo
}
#######################################
#######################################
function B14 {
	umount -a
}
#######################################
#######################################
function B15 {
	bootlist -om normal | tee /tmp/bootlist_normal_prev
}
#######################################
#######################################
function B16 {
	/bin/echo "Prepare your console access:"
	/bin/echo "From SAPRMT01 or SAPRMT02:"
	/usr/local/scripts/list_hmc.sh -q server=`uname -n` -C
	#sudo -u saunix ssh -q saprmt01 /usr/local/scripts/find_frame_info.sh -l mhae770a -h `uname -n` -c | grep -v '#'
}
#######################################
#######################################
function B17 {
	/bin/echo "Now is time to reboot, start by running \"shutdown -Fr\""
	/bin/echo "From SAPRMT01 or SAPRMT02, you can control the lpar console from the HMC with these commands:"
	/usr/local/scripts/list_hmc.sh -q name=`uname -n` -AD
	#sudo -u saunix ssh -q saprmt01 /usr/local/scripts/find_frame_info.sh -l mhae770a -h `uname -n` -AD | grep -v '#'
	REBOOT
}
#######################################
#######################################
function B18 {
	OSLEVEL=$(oslevel -s)
	/bin/echo $OSLEVEL | grep -q $TARGETOSLEVEL && /bin/echo "\nok, oslevel at $TARGETOSLEVEL, you can restart the apps\n" || /bin/echo "\nFAIL! Not ok, please verify with \"oslevel -sl $TARGETOSLEVEL\"\nEXIT"
}
#######################################
#######################################
function B19 {
	/bin/echo "Executing lppchk -vm3: \c"
	if ! lppchk -vm3 ; then /bin/echo "Something went wrong, exiting"; EXIT; else /bin/echo ok; fi
}
#######################################
#######################################
function B20 {
	(cd /central/depot/rpm/; ./install_gnu)
	(cd /; ln -fs /opt/freeware/sbin/lsof /usr/bin/lsof)
}
#######################################
#######################################
function B21 {
	if [ "$(lparstat -i|grep 'Memory Expansion Factor' | awk '{print $NF}')" != "1" ]
	then
		/bin/echo "verify whether IBM  LPAR Mgmt Group has set AME to active with Memory Expansion Factor 1 (no compression)."
		/bin/echo "Currently, \"lparstat -i|grep 'Memory Expansion Factor'\" doesn' show \"1\""
		/bin/echo "Press any key to continue..."
		read -n 1
	else
		/bin/echo "AME active with Memory Expansion Factor 1 (no compression)"
	fi
}
#######################################
#######################################
function B22 {
	BAD=no
	for VSCSI in `lsdev| grep vscsi| grep Avail | awk '{print $1}'`
	do 
		if [ "$(lsattr -El $VSCSI| grep vscsi_err_recov |awk '{print $2}')" = "fast_fail" ] && [ "$(lsattr -El $VSCSI| grep vscsi_path_to |awk '{print $2}')" = "30" ]
		then 
			/bin/echo ok 
		else 
			/bin/echo $VSCSI has bad settings
			/bin/echo Verify with lsattr -El $VSCSI
			/bin/echo "You should have vscsi_err_recov -> fast_fail"
			/bin/echo "You should have vscsi_path_     -> 30"
			/bin/echo "Here are the right commands:\n"
			cat <<-'EOPART'
				lsdev | grep vscsi | grep Avail | awk '{print $1}' | while read line ; do echo $line ; chdev -l $line -a vscsi_err_recov=fast_fail -a vscsi_path_to=30 -P ; done
				lsdev | grep disk | grep Virtu | grep Avail | awk '{print $1}' | while read line ; do echo $line ; chdev -l $line -a hcheck_interval=60 -P ; done
			EOPART
			/bin/echo
			BAD=yes
		fi
	done
	for HDISK in `lsdev| grep Virt| grep hdisk | grep Avail | awk '{print $1}'`
	do 
		if [ "$(lsattr -El $HDISK| grep hcheck_interval |cut -d' ' -f2)" = "60" ]
		then 
			/bin/echo ok 
		else 
			/bin/echo $HDISK has bad settings
			/bin/echo Verify with lsattr -El $HDISK
			/bin/echo You can re-execute step B13
			BAD=yes
		fi 
	done
	if ! ioo -o pd_npages | tee /tmp/pd_npages_old_`date +%Y-%m-%d-%H-%M` | grep -q 32768
	then
		/bin/echo "Setting in ioo -o pd_npages not good (should be 32768)"
		BAD=yes
	fi
	if [ "$BAD" = "yes" ]
	then
		/bin/echo "Press any key to continue..."
		read -n 1
		/bin/echo
	fi
}
#######################################
#######################################
function B23 {
	/bin/echo "y\ny\ny\ny\ny" > /tmp/yes
	if ! vmo -p -o minperm%=3 -o maxperm%=90 -o maxclient%=90 -o lru_file_repage=0 -o strict_maxclient=1 -o strict_maxperm=0 -o minfree=960 -o maxfree=1088 < /tmp/yes
	then
		BAD=yes
	fi
	if ! chdev -l sys0 -a maxpout=513 -a minpout=256
	then
		BAD=yes
	fi
	if [ "$BAD" = "yes" ]
	then
		/bin/echo
		/bin/echo "Press any key to continue..."
		read -n 1
		/bin/echo
	fi
	BAD=no
	while IFS=: read SETTING VALUE
	do
		case $SETTING in
			'minperm%')
				GOOD=3
				if [ "$VALUE" != "$GOOD" ]; then echo BAD=yes; $SETTING not right, should be $GOOD; fi
				;;
			'maxperm%')
				GOOD=90
				if [ "$VALUE" != "$GOOD" ]; then echo BAD=yes; $SETTING not right, should be $GOOD; fi
				;;
			'maxclient%')
				GOOD=90
				if [ "$VALUE" != "$GOOD" ]; then echo BAD=yes; $SETTING not right, should be $GOOD; fi
				;;
			'lru_file_repage')
				GOOD=0
				if [ "$VALUE" != "$GOOD" ]; then echo BAD=yes; $SETTING not right, should be $GOOD; fi
				;;
			'strict_maxclient')
				GOOD=1
				if [ "$VALUE" != "$GOOD" ]; then echo BAD=yes; $SETTING not right, should be $GOOD; fi
				;;
			'strict_maxperm')
				GOOD=0
				if [ "$VALUE" != "$GOOD" ]; then echo BAD=yes; $SETTING not right, should be $GOOD; fi
				;;
			'minfree')
				GOOD=960
				if [ "$VALUE" != "$GOOD" ]; then echo BAD=yes; $SETTING not right, should be $GOOD; fi
				;;
			'maxfree')
				GOOD=1088
				if [ "$VALUE" != "$GOOD" ]; then echo BAD=yes; $SETTING not right, should be $GOOD; fi
				;;
		esac
	done < <(vmo -FL| grep -E  "^minfree|^maxfree|^minperm%|^maxperm%|^maxclient%|^lru_file_repage|^strict_maxclient|^strict_maxperm " | awk '{print $1":"$2}')
	while IFS=: read SETTING VALUE
	do
		case $SETTING in
			'maxpout')
				if [ "$VALUE" != "513" ] && [ "$VALUE" != "8192" ]; then echo BAD=yes; $SETTING not right, should be 513 or 8192; fi
				;;
			'minpout')
				if [ "$VALUE" != "256" ] && [ "$VALUE" != "4096" ]; then echo BAD=yes; $SETTING not right, should be 256 or 4096; fi
				;;
		esac
	done < <(lsattr -El sys0 -a maxpout -a minpout | awk '{print $1":"$2}')
	if [ "$BAD" = "yes" ]
	then
		/bin/echo
		/bin/echo "Press any key to continue..."
		read -n 1
		/bin/echo
	fi
}
#######################################
#######################################
function B24 {
	/usr/sbin/chedition -s
}
#######################################
#######################################
function B25 {
	if [ -f /usr/es/sbin/cluster/utilities/cltopinfo ]
	then
		/bin/echo "This server to have HACMP, check DMS and RG timeout and monitor interval"
		/bin/echo "Refer to Change of DMS timeout and RG monitor interval on the running HACMP.doc"
		/bin/echo "http://ecm.aero.bombardier.net:84/webtop/drl/objectId/0901260f801a2fdf"
	else
		/bin/echo "No HACMP on this server"
	fi
}
#######################################
#######################################
function B26_1 {
	DIR=/usr/sap/DAA/SMDA98/SMDAgent/applications.config/com.sap.smd.agent.application.wily/BytecodeAgent/ISAGENT*/wily/connectors
	if [ -d $DIR ]
	then
		cd $DIR
		if ! java -Dcom.wily.autoprobe.prependToJVMRuntimePath=/usr/java14_64/jre/lib/jclSC14/classes.zip -jar CreateAutoProbeConnector.jar -jvm /usr/java14_64 -output AutoProbeConnector.jar; then EXIT; fi
	else
		/bin/echo "Dir $DIR doesn't exit"
	fi
}
#######################################
#######################################
function B26 {
	read -n 1 -p "Are the applications down? [y/n]: "
	/bin/echo
	if [ "$REPLY" = "y" ]
   then
		/bin/echo "Please start applications"
		/bin/echo
		/bin/echo "Re-execute this script after you are done."
		/bin/echo
		EXIT
	fi
}
#######################################
#######################################
function B27 {
	/bin/echo "Please Remove Tivoli maintenance filters"
}
#######################################
#######################################
function B28 {
	/bin/echo "Running syscheck"
	/usr/local/scripts/syscheck
}
#######################################
